/**
 * @author Fang Lu, fanglupku@gmail.com
 * 2011-3-20
 */
package code.extract.training;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;

import code.extract.FileDealer;

import net.PageFetcher;

public class PageCrawlThread implements Runnable{
	private String savepath = ".";
	private int id = -1;
	private static int threadNum = -1;
	private static PageLinkList linkList = null;
	
	public PageCrawlThread(String savepath, int id) {
		this.savepath = savepath;
		this.id = id;
	}
	
	public static void initPageLinkList (PageLinkList linkList){
		PageCrawlThread.linkList = linkList;
	}
	
	public static void initThreadNum(int threadNum){
		PageCrawlThread.threadNum = threadNum;
	}
	
	@Override
	public void run() {
		HashSet<String> keywordSet = linkList.getKeywordSet();
		HashSet<String> crawledKeywordSet = linkList.getCrawledKeywordSet();
		Iterator<String> keyword_itor = keywordSet.iterator();
		System.out.println(id + ": start");
		for (int i=0; i<keywordSet.size(); ++i){
			String thisKeyword = keyword_itor.next();
			if (i % threadNum == id && !crawledKeywordSet.contains(thisKeyword)){
				File saveDir = new File(savepath + File.separatorChar + i);
				if (!saveDir.exists()){
					saveDir.mkdirs();
				}
				ArrayList<String> thisLinkList = linkList.getKeywordLinkList().get(thisKeyword);
				if (thisLinkList == null){
					System.err.println("empty link list: keyword=" + thisKeyword);
					continue;
				}
				for (int j=0; j<thisLinkList.size(); ++j){
					String thisLink = thisLinkList.get(j);
					if (crawledKeywordSet.contains(thisLinkList)){
						continue;
					}
					String pageContent = PageFetcher.fetchURL(thisLink);
					String filename = savepath + File.separatorChar + i + File.separatorChar + i + "_" + j + ".txt";
					try {
						FileDealer.saveFile(pageContent, filename);
					} catch (IOException e) {
						// TODO Auto-generated catch block
						e.printStackTrace();
					}
					linkList.addCrawledLink(thisLink);
				}
				linkList.addCrawledKeyword(thisKeyword);
//				linkList.addCrawledLinklist(thisLinkList);
			}
		}
		System.out.println(id + ": end");
	}
}
